#include "DictProducer.h"

#include <fstream>
#include <iostream>
#include <sstream>

using std::cerr;
using std::cout;
using std::ifstream;
using std::istringstream;
using std::ofstream;
using std::ostringstream;

// 用于初始化字典对象，打开语料文件的路径
DictProducer::DictProducer(Configuration *conf) {
    _cuttor = new SplitToolCppJieba(conf); // 触发多态

    // 读取词典路径
    string eng_text_path = conf->get("paths", "data") + "/The_Holy_Bible.txt"; // eng
    string eng_stop_path = conf->get("paths", "data") + "/stop_words_eng.txt";
    string chs_text_path = conf->get("paths", "data") + "/C3-Art0019.txt"; // cns
    string chs_stop_path = conf->get("paths", "data") + "/stop_words_zh.txt";

    // 调试代码：测试路径
    cout << "test: \n";
    cout << "eng_text_path = " << eng_text_path << "\n";
    cout << "eng_stop_path = " << eng_stop_path << "\n";
    cout << "chs_text_path = " << chs_text_path << "\n";
    cout << "chs_stop_path = " << chs_stop_path << "\n";

    // 添加至路径
    _files.push_back(eng_text_path); // 0
    _files.push_back(eng_stop_path); // 1
    _files.push_back(chs_text_path); // 2
    _files.push_back(chs_stop_path); // 3
};

DictProducer::~DictProducer() {
    if (_cuttor) {
        delete _cuttor;
        _cuttor = nullptr;
    }
}

void DictProducer::buildEnDict() {
    // 打开文件流
    ifstream ifs_eng_text(_files[0]);
    ifstream ifs_eng_stop(_files[1]);

    // 清洗
    // 读取文件流的所有数据
    ostringstream iss_eng_text;
    iss_eng_text << ifs_eng_text.rdbuf();
    string eng_text = iss_eng_text.str();

    ostringstream iss_eng_stop;
    iss_eng_stop << ifs_eng_stop.rdbuf();
    string eng_stop = iss_eng_stop.str();

    ifs_eng_text.close();
    ifs_eng_stop.close();

    // 第一步：对iss的预处理
    for (auto &e : eng_text) {
        if (!isalpha(e)) {
            e = ' ';
        } else if (isupper(e)) {
            e = tolower(e);
        }
    }

    for (auto &e : eng_stop) {
        if (!isalpha(e)) {
            e = ' ';
        } else if (isupper(e)) {
            e = tolower(e);
        }
    }

    // 第二部分处理,导入map字典
    dealEngWorld(eng_text, eng_stop);
};

void DictProducer::buildCnDict() {
    // 创建中文词典
    // 打开文件流
    ifstream ifs_chs_text(_files[2]);
    ifstream ifs_chs_stop(_files[3]);

    // 清洗
    // 读取文件流的所有数据
    ostringstream iss_chs_text; // chs
    iss_chs_text << ifs_chs_text.rdbuf();
    string chs_text = iss_chs_text.str();

    ostringstream iss_chs_stop; // chs
    iss_chs_stop << ifs_chs_stop.rdbuf();
    string chs_stop = iss_chs_stop.str();

    ifs_chs_text.close();
    ifs_chs_stop.close();
    // 第一步：对iss的预处理
    chs_text.erase(std::remove(chs_text.begin(), chs_text.end(), '\n'), chs_text.end());
    chs_text.erase(std::remove(chs_text.begin(), chs_text.end(), '\r'), chs_text.end());

    chs_stop.erase(std::remove(chs_stop.begin(), chs_stop.end(), '\r'), chs_stop.end());

    // 第二部分处理,导入map字典，统计词频
    dealChsWorld(chs_text, chs_stop);
};

size_t nBytesCode(const char ch);
// 创建索引
void DictProducer::createIndex() {
    // 英文词典
    // 需要一个循环来遍历每一个单词
    int idx_eng = 0;
    for (auto pair : _eng_text) {
        // 先将map字典转为vector<pair<string, int>> 压缩操作
        // 遍历单词的每个字母
        _eng_dict.push_back(pair);
        for (auto c : pair.first) {
            // 存储单词位置索引,map<string, set<int>>
            _eng_index[string(1, c)].insert(idx_eng);
        }
        ++idx_eng;
    }

    // 调试代码：测试路径
    cout << "test: \n";
    cout << "ofs_eng_idx size is " << _eng_index.size() << "\n";

    // 创建中文词典索引
    int idx_chs = 0;
    for (auto pair : _chs_text) {
        _chs_dict.push_back(pair);
        string str = pair.first;
        for (size_t idx = 0; idx < str.size();) {
            size_t len = nBytesCode(str[0]);
            _chs_index[str.substr(idx, len)].insert(idx_chs);
            idx += 3;
        }
        ++idx_chs;
    }
    cout << "_chs_index size is " << _chs_index.size() << "\n";
};

// 保存字典文件
void DictProducer::store(const string &path) {
    string eng_filename = path + "/dict_eng.dat";
    string eng_idx_filename = path + "/dict_eng_index.dat";
    string chs_filename = path + "/dict_chs.dat";
    string chs_idx_filename = path + "/dict_chs_index.dat";

    // 保存英文词典
    ofstream ofs_eng(eng_filename);
    for (auto &ele : _eng_text) {
        ofs_eng << ele.first << " " << ele.second << "\n";
    }

    ofstream ofs_eng_idx(eng_idx_filename);
    for (auto &pair : _eng_index) {
        ofs_eng_idx << pair.first << " ";

        for (auto &e : pair.second) {
            ofs_eng_idx << e << " ";
        }
        ofs_eng_idx << "\n";
    }

    // 保存中文词典
    ofstream ofs_chs(chs_filename);
    for (auto &pair : _chs_text) {
        ofs_chs << pair.first << " " << pair.second << "\n";
    }

    ofstream ofs_chs_idx(chs_idx_filename);
    for (auto &pair : _chs_index) {
        ofs_chs_idx << pair.first << " ";

        for (auto &e : pair.second) {
            ofs_chs_idx << e << " ";
        }
        ofs_chs_idx << "\n";
    }

    // 关闭文件流
    ofs_eng.close();
    ofs_eng_idx.close();
    ofs_chs.close();
    ofs_chs_idx.close();
};

// 统计词频
void DictProducer::dealEngWorld(const string &eng_text, const string &eng_stop) {
    // 读取stop_eng
    istringstream iss_stop(eng_stop);
    string stop_word;
    while (iss_stop >> stop_word) {
        _eng_stop.insert(stop_word);
    }

    // 读取eng
    istringstream iss_eng(eng_text);
    string word;
    while (iss_eng >> word) {
        if (_eng_stop.count(word) == 0) {
            ++_eng_text[word];
        }
    }

    cout << "eng_text size is " << _eng_text.size() << "\n";
    cout << "eng_stop size is " << _eng_stop.size() << "\n";
}

void DictProducer::dealChsWorld(const string &chs_text, const string &chs_stop) {
    // 利用结巴对象进行分割，将其放入map
    // 加载停用词
    string stop_word;
    istringstream iss_chs_stop(chs_stop);
    while (iss_chs_stop >> stop_word) {
        _chs_stop.insert(stop_word);
    }

    // 加载中文词典
    vector<string> words = _cuttor->cut(chs_text);
    for (auto &w : words) {
        size_t nByte = nBytesCode(w[0]);
        if (!_chs_stop.count(w) && nByte == 3) {
            ++_chs_text[w];
        }
    }
}

size_t nBytesCode(const char ch) {
    if (ch & (1 << 7)) {
        int nBytes = 1;
        for (int idx = 0; idx != 6; ++idx) {
            if (ch & (1 << (6 - idx))) {
                ++nBytes;
            } else
                break;
        }
        return nBytes;
    }
    return 1;
}